package com.wd.live.platform;

import java.util.LinkedList;
import java.util.List;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectors;
import us.codecraft.webmagic.selector.XpathSelector;

public class DouyuRepoPageProcessor extends AbstractProcessor {
	
	private Pipeline pipeline;
	// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
	  private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);
	  public DouyuRepoPageProcessor(Pipeline pipeline){
		  this.pipeline = pipeline;
	  }
	  private DouyuRepoPageProcessor(){
	  }
	  public void process(Page page){
		  List<String> list = page.getHtml().xpath("//ul[@id='live-list-contentbox']/li/a/div[@class='mes']/html()").all();
		  List<String[]> result  = new LinkedList<String[]>();
		  for(String str : list){
			  String[] array = new String[4];
			  XpathSelector xpathSelector = Selectors.xpath("//div/span/html()");
			  String gameName = new Html(xpathSelector.selectList(str), false).toString();
			  array[0] = gameName;
			  
			  xpathSelector = Selectors.xpath("//p/span/html()");
			  String userName = new Html(xpathSelector.selectList(str), false).toString();
			  array[1] = userName;
			  
			  xpathSelector = Selectors.xpath("//p/span[@class='dy-num fr']/html()");
			  String total = new Html(xpathSelector.selectList(str), false).toString();
			  array[2] = total;
			  
			  array[3] = "斗魚";
			  
			  result.add(array);
		  }
		  
		  page.putField("listDate", result);
	  }

	  public Site getSite() {
	    return site;
	  }
	  
	
	  public void execute() {
	    Spider.create(new DouyuRepoPageProcessor())
	        //从"https://github.com/code4craft"开始抓
	        .addUrl("http://www.douyutv.com/directory/all").addPipeline(pipeline)
	        //开启5个线程抓取
	        .thread(5)
	        //启动爬虫
	        .run();
	  }
	}